{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ac323b7b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-16T09:57:56.565229Z",
     "start_time": "2022-05-16T09:57:55.848528Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dcfa9886",
   "metadata": {},
   "source": [
    "## 原始数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ee3873ec",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T06:56:44.455527Z",
     "start_time": "2022-05-13T06:56:44.442837Z"
    }
   },
   "outputs": [],
   "source": [
    "# 一般使用特殊类型 NaN 代表缺失值，\n",
    "# 可以用 Numpy 可定义它np.NaN/np.nan。\n",
    "# 在 Pandas 1.0 以后实验性地使用一个标量 pd.NA 来代表\n",
    "\n",
    "df = pd.DataFrame({'Unnamed:0': [0,1,2,3,4,5,6],\n",
    "                  'Date': ['04/04/2021','april 17th,2021','2021-08-21','05/02/2021','05/12/2021','2021-10-11',np.nan],\n",
    "                  'Name': ['doe,john','Doe,Jane','smith,Adam','Tuck,matt','James,Ben',np.nan,np.nan],\n",
    "                  'Payment': ['$100,50','$78.50','$65','$120',np.nan,'$100',np.nan],\n",
    "                  'Note': ['Unhappy!','?Satidfied','Neutral','Unhappy-','Neutral0','Neutral',np.nan],\n",
    "                  'Remark':[np.nan,np.nan,np.nan,np.nan,np.nan,np.nan,np.nan]},\n",
    "#                  index = [0,1,2,3,4,5]\n",
    "                 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ea419147",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T06:56:45.271471Z",
     "start_time": "2022-05-13T06:56:45.247002Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed:0</th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "      <th>Remark</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>05/12/2021</td>\n",
       "      <td>James,Ben</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Neutral0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>2021-10-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$100</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed:0             Date        Name  Payment        Note  Remark\n",
       "0          0       04/04/2021    doe,john  $100,50    Unhappy!     NaN\n",
       "1          1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied     NaN\n",
       "2          2       2021-08-21  smith,Adam      $65     Neutral     NaN\n",
       "3          3       05/02/2021   Tuck,matt     $120    Unhappy-     NaN\n",
       "4          4       05/12/2021   James,Ben      NaN    Neutral0     NaN\n",
       "5          5       2021-10-11         NaN     $100     Neutral     NaN\n",
       "6          6              NaN         NaN      NaN         NaN     NaN"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ebbbf79f",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2022-05-13T02:38:22.911Z"
    }
   },
   "outputs": [],
   "source": [
    "# 让我们看看可以做些什么来使这个数据集变得干净。\n",
    "\n",
    "# 第一列是多余的，应该删除；\n",
    "# Remark全为NAN,全部删除\n",
    "# 索引6为NAN,全部删除\n",
    "# Date 没有标准；\n",
    "# Name 写成姓氏、名字，并有大写和小写字母；\n",
    "# Payment 代表一个数量，但它们显示为字符串，需要处理；\n",
    "# 在 Note 中，有一些非字母数字应该被删除;"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a697ee1e",
   "metadata": {},
   "source": [
    "## 删除多余数据"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06301c78",
   "metadata": {},
   "source": [
    "### 肯定会完全删除的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a630b48b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 冗余的列\n",
    "# 全为NAN的列\n",
    "# 全为NAN的行"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3240f13d",
   "metadata": {},
   "source": [
    "#### 删除冗余列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fda5cb5b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T06:57:00.538867Z",
     "start_time": "2022-05-13T06:57:00.521435Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "      <th>Remark</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05/12/2021</td>\n",
       "      <td>James,Ben</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Neutral0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2021-10-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$100</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note  Remark\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!     NaN\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied     NaN\n",
       "2       2021-08-21  smith,Adam      $65     Neutral     NaN\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-     NaN\n",
       "4       05/12/2021   James,Ben      NaN    Neutral0     NaN\n",
       "5       2021-10-11         NaN     $100     Neutral     NaN\n",
       "6              NaN         NaN      NaN         NaN     NaN"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 删除列是使用 drop 函数的简单操作。\n",
    "# 除了写列名外，我们还需要指定轴参数的值，因为 drop 函数用于删除行和列。 \n",
    "# 最后，我们可以使用 inplace 参数来保存更改。\n",
    "\n",
    "# 指定列名直接删除\n",
    "df.drop('Unnamed:0', axis=1, inplace=True) \n",
    "df "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b41c79a",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2022-05-13T03:50:40.836Z"
    }
   },
   "source": [
    "#### 删除列全为空的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b65a4fdc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T06:57:14.811855Z",
     "start_time": "2022-05-13T06:57:14.800561Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05/12/2021</td>\n",
       "      <td>James,Ben</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Neutral0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2021-10-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$100</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied\n",
       "2       2021-08-21  smith,Adam      $65     Neutral\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-\n",
       "4       05/12/2021   James,Ben      NaN    Neutral0\n",
       "5       2021-10-11         NaN     $100     Neutral\n",
       "6              NaN         NaN      NaN         NaN"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dropna删除列全为空的\n",
    "df.dropna(axis=1, how='all', inplace=True)\n",
    "df "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8732d6fc",
   "metadata": {},
   "source": [
    "#### 删除行全为空的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "42be1744",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T06:57:16.718843Z",
     "start_time": "2022-05-13T06:57:16.695233Z"
    }
   },
   "outputs": [],
   "source": [
    "# dropna删除行全为空的\n",
    "df.dropna(axis=0,  how = 'all',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "9b22f342",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T06:57:17.843816Z",
     "start_time": "2022-05-13T06:57:17.814877Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05/12/2021</td>\n",
       "      <td>James,Ben</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Neutral0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2021-10-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$100</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied\n",
       "2       2021-08-21  smith,Adam      $65     Neutral\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-\n",
       "4       05/12/2021   James,Ben      NaN    Neutral0\n",
       "5       2021-10-11         NaN     $100     Neutral"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77152b68",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T02:53:46.715382Z",
     "start_time": "2022-05-13T02:53:46.709874Z"
    }
   },
   "source": [
    " ### 对于行或者列中存在NAN的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "236903b5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T07:06:00.670073Z",
     "start_time": "2022-05-13T07:06:00.593186Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05/12/2021</td>\n",
       "      <td>James,Ben</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Neutral0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2021-10-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$100</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied\n",
       "2       2021-08-21  smith,Adam      $65     Neutral\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-\n",
       "4       05/12/2021   James,Ben      NaN    Neutral0\n",
       "5       2021-10-11         NaN     $100     Neutral"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'Unnamed:0': [0,1,2,3,4,5,6],\n",
    "                  'Date': ['04/04/2021','april 17th,2021','2021-08-21','05/02/2021','05/12/2021','2021-10-11',np.nan],\n",
    "                  'Name': ['doe,john','Doe,Jane','smith,Adam','Tuck,matt','James,Ben',np.nan,np.nan],\n",
    "                  'Payment': ['$100,50','$78.50','$65','$120',np.nan,'$100',np.nan],\n",
    "                  'Note': ['Unhappy!','?Satidfied','Neutral','Unhappy-','Neutral0','Neutral',np.nan],\n",
    "                  'Remark':[np.nan,np.nan,np.nan,np.nan,np.nan,np.nan,np.nan]},\n",
    "#                  index = [0,1,2,3,4,5]\n",
    "                 )\n",
    "df.drop('Unnamed:0', axis=1, inplace=True)  \n",
    "df.dropna(axis=1,how='all', inplace=True)\n",
    "df.dropna(axis=0,how='all',inplace=True)\n",
    "df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f235ebc",
   "metadata": {},
   "source": [
    "#### 判断是否为空值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "c2bfc0f3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T06:40:57.226909Z",
     "start_time": "2022-05-13T06:40:57.166736Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2    False\n",
       "3    False\n",
       "4    False\n",
       "5     True\n",
       "Name: Name, dtype: bool"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 判断是否为空值\n",
    "df['Name'].notna()   # 不是空值,显示True\n",
    "df['Name'].isna()  # 是空值,显示True\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34cb9573",
   "metadata": {},
   "source": [
    "#### 筛选出不为空的值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0ab3b638",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T06:49:15.669715Z",
     "start_time": "2022-05-13T06:49:15.642047Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2021-10-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$100</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Date Name Payment     Note\n",
       "5  2021-10-11  NaN    $100  Neutral"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['Name'].notna()] # 筛选出Name不为空的数据\n",
    "df[df['Name'].isna()]  # 筛选出Name为空的数据"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feb3e5c5",
   "metadata": {},
   "source": [
    "#### 删除包含NAN的行列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "be57408c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied\n",
       "2       2021-08-21  smith,Adam      $65     Neutral\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 删除行列可以通过以下方式: \n",
    "df.dropna(inplace=True) # 一行中有一个空NaN就删除\n",
    "# df.dropna(axis='columns') # 只保留全有值的列\n",
    "# df.dropna(how='all') # 行或列全没值才删除\n",
    "# df.dropna(thresh=2) # 至少有两个空值时才删除\n",
    "# df.dropna(inplace=True) # 删除并生效替换\n",
    "df "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd6e4629",
   "metadata": {},
   "source": [
    "##### 删除所有存在NAN的行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "db01f469",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed:0</th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "      <th>Remark</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Unnamed:0, Date, Name, Payment, Note, Remark, name]\n",
       "Index: []"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "923a8f38",
   "metadata": {},
   "source": [
    "##### 删除指定列为空的行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c64dbd10",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T07:00:56.165113Z",
     "start_time": "2022-05-13T07:00:56.131447Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied\n",
       "2       2021-08-21  smith,Adam      $65     Neutral\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 删除Name中包含NAN的行\n",
    "df.dropna(axis=0,subset=[\"Name\"], inplace= True)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62e6e629",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T06:54:47.061692Z",
     "start_time": "2022-05-13T06:54:47.037864Z"
    }
   },
   "source": [
    "#### NAN填充"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0ba7a713",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T07:06:42.285149Z",
     "start_time": "2022-05-13T07:06:42.260983Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied\n",
       "2       2021-08-21  smith,Adam      $65     Neutral\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fillna(x) 可以将缺失值填充指定的值。\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "796ae659",
   "metadata": {},
   "source": [
    "##### 填充固定值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bb94da42",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T07:11:55.634352Z",
     "start_time": "2022-05-13T07:11:55.601499Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied\n",
       "2       2021-08-21  smith,Adam      $65     Neutral\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 填充(4, Payment为例)\n",
    "\n",
    "df.fillna(0)# 填充为0\n",
    "\n",
    "df.fillna('$90') # 填充指定字符\n",
    "\n",
    "df['Payment'].fillna('$90') # 指定字段(列),填充指定字符\n",
    "\n",
    "# df.fillna(xxx, inplace=Ture) # 填充内容生效\n",
    "\n",
    "df.fillna(0, limit=1)  # 只替换第一个\n",
    "\n",
    "# 不同列替换不同的值\n",
    "values = {'Name': 'James,matt', 'Payment': '$90'}\n",
    "df.fillna(value=values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18f868ca",
   "metadata": {},
   "source": [
    "##### 填充已有的值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d0d0bf29",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pad / ffill：向前填充\n",
    "# bfill / backfill：向后填充"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c5106f0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied\n",
       "2       2021-08-21  smith,Adam      $65     Neutral\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# bfill / backfill,使用上一个有效值填充,作用一样\n",
    "df.fillna(method='backfill')\n",
    "df.fillna(method='bfill')\n",
    "\n",
    "df.fillna(method='bfill').head(1).iloc[0] # 填充第一个非空值\n",
    "\n",
    "\n",
    "# pad / ffill, 把当前值广播到后边的缺失值,作用一样\n",
    "df.fillna(method='pad')\n",
    "df.fillna(method='ffill')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13ff70a8",
   "metadata": {},
   "source": [
    "##### 填充计算值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5d198f22",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\10191977\\AppData\\Local\\Temp\\ipykernel_18416\\3434118128.py:2: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  df.fillna(df.mean())\n",
      "C:\\Users\\10191977\\AppData\\Local\\Temp\\ipykernel_18416\\3434118128.py:5: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  df.fillna(df.mean()['B':'C'])\n",
      "C:\\Users\\10191977\\AppData\\Local\\Temp\\ipykernel_18416\\3434118128.py:6: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  df.fillna(df.mean()['B':'C'])\n",
      "C:\\Users\\10191977\\AppData\\Local\\Temp\\ipykernel_18416\\3434118128.py:9: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  df.where(pd.notna(df), df.mean(), axis='columns')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied\n",
       "2       2021-08-21  smith,Adam      $65     Neutral\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 填充列的平均值\n",
    "df.fillna(df.mean())\n",
    "\n",
    "# 对指定列填充平均值\n",
    "df.fillna(df.mean()['B':'C'])\n",
    "df.fillna(df.mean()['B':'C'])\n",
    "\n",
    "# 填充列的平均值，另外一个方法\n",
    "df.where(pd.notna(df), df.mean(), axis='columns')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfd2c7f2",
   "metadata": {},
   "source": [
    "## 统一格式"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39d96d5c",
   "metadata": {},
   "source": [
    "### 统一时间格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "255c4fc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'Unnamed:0': [0,1,2,3,4,5,6],\n",
    "                  'Date': ['04/04/2021','april 17th,2021','2021-08-21','05/02/2021','05/12/2021','2021-10-11',np.nan],\n",
    "                  'Name': ['doe,john','Doe,Jane','smith,Adam','Tuck,matt','James,Ben',np.nan,np.nan],\n",
    "                  'Payment': ['$100,50','$78.50','$65','$120',np.nan,'$100',np.nan],\n",
    "                  'Note': ['Unhappy!','?Satidfied','Neutral','Unhappy-','Neutral0','Neutral',np.nan],\n",
    "                  'Remark':[np.nan,np.nan,np.nan,np.nan,np.nan,np.nan,np.nan]},\n",
    "#                  index = [0,1,2,3,4,5]\n",
    "                 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e04c85a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed:0</th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "      <th>Remark</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>05/12/2021</td>\n",
       "      <td>James,Ben</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Neutral0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>2021-10-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$100</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed:0             Date        Name  Payment        Note  Remark\n",
       "0          0       04/04/2021    doe,john  $100,50    Unhappy!     NaN\n",
       "1          1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied     NaN\n",
       "2          2       2021-08-21  smith,Adam      $65     Neutral     NaN\n",
       "3          3       05/02/2021   Tuck,matt     $120    Unhappy-     NaN\n",
       "4          4       05/12/2021   James,Ben      NaN    Neutral0     NaN\n",
       "5          5       2021-10-11         NaN     $100     Neutral     NaN\n",
       "6          6              NaN         NaN      NaN         NaN     NaN"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "26b254d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop('Unnamed:0', axis=1, inplace=True)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "46ccf37e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(axis=1,how='all', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "014daded",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05/12/2021</td>\n",
       "      <td>James,Ben</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Neutral0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2021-10-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$100</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied\n",
       "2       2021-08-21  smith,Adam      $65     Neutral\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-\n",
       "4       05/12/2021   James,Ben      NaN    Neutral0\n",
       "5       2021-10-11         NaN     $100     Neutral\n",
       "6              NaN         NaN      NaN         NaN"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "777c1830",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5078e31d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied\n",
       "2       2021-08-21  smith,Adam      $65     Neutral\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ae756b3f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T02:42:33.786676Z",
     "start_time": "2022-05-13T02:42:33.780804Z"
    }
   },
   "outputs": [],
   "source": [
    "# 我们有多种选择将日期值转换为适当的格式。一种更简单的方法是使用 astype 函数来更改列的数据类型。 \n",
    "# 它能够处理范围广泛的值并将它们转换为整洁、标准的日期格式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "631a872e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T02:55:10.711695Z",
     "start_time": "2022-05-13T02:55:10.701327Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\10191977\\AppData\\Local\\Temp\\ipykernel_1356\\3035710072.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['Date'] = df['Date'].astype('datetime64[ns]')\n"
     ]
    }
   ],
   "source": [
    "df['Date'] = df['Date'].astype('datetime64[ns]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0d2bd34b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T02:55:11.363118Z",
     "start_time": "2022-05-13T02:55:11.329646Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-04-04</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-04-17</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-05-02</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Date        Name  Payment        Note\n",
       "0 2021-04-04    doe,john  $100,50    Unhappy!\n",
       "1 2021-04-17    Doe,Jane   $78.50  ?Satidfied\n",
       "2 2021-08-21  smith,Adam      $65     Neutral\n",
       "3 2021-05-02   Tuck,matt     $120    Unhappy-"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "80261fd9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    20191209\n",
       "1    20191202\n",
       "Name: year-month-day, dtype: object"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2\n",
    "df = pd.DataFrame(['2019-12-09', '2019-12-02'], columns=[\"date\"])\n",
    "df \n",
    "\n",
    "# 转换为时间\n",
    "df[\"date\"] = pd.to_datetime(df[\"date\"])\n",
    "# 获取年月日\n",
    "df[\"year-month-day\"] = df[\"date\"].apply(lambda x: x.strftime(\"%Y%m%d\"))\n",
    "df[\"year-month-day\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31b85a2f",
   "metadata": {},
   "source": [
    "### 统一姓名格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7c80972b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T02:55:12.548308Z",
     "start_time": "2022-05-13T02:55:12.538502Z"
    }
   },
   "outputs": [],
   "source": [
    "# 首先我们应该用所有大写或小写字母来表示它们。\n",
    "# 另一种选择是将它们大写（即只有首字母是大写的）；\n",
    "# 切换姓氏和名字的顺序；"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fa8a249a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-04-04</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-04-17</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-05-02</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Date        Name  Payment        Note\n",
       "0 2021-04-04    doe,john  $100,50    Unhappy!\n",
       "1 2021-04-17    Doe,Jane   $78.50  ?Satidfied\n",
       "2 2021-08-21  smith,Adam      $65     Neutral\n",
       "3 2021-05-02   Tuck,matt     $120    Unhappy-"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "c7aa3f1d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T03:04:58.292147Z",
     "start_time": "2022-05-13T03:04:58.268167Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    john\n",
       "1    Jane\n",
       "2    Adam\n",
       "3    matt\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将名字和姓氏分开\n",
    "df['Name'].str.split(',',expand = True)\n",
    "\n",
    "# df['Name'].str.split(expand = True)\n",
    "# df['Name'].str.split(',',expand = True)[1]\n",
    "# df['Name'].str.split(',',expand = True)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "4f4c701a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T03:05:17.097958Z",
     "start_time": "2022-05-13T03:05:17.081488Z"
    }
   },
   "outputs": [],
   "source": [
    "# 取第二列与第一列结合起来，中间有一个空格。\n",
    "# 最后一步是使用 lower 函数将字母转换为小写。\n",
    "\n",
    "df['Name'] = (df['Name'].str.split(',',expand = True)[1] + ' ' + df['Name'].str.split(',',expand = True)[0]).str.lower()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "269ffc70",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T03:05:19.548154Z",
     "start_time": "2022-05-13T03:05:19.513026Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-04-04</td>\n",
       "      <td>john doe</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-04-17</td>\n",
       "      <td>jane doe</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>adam smith</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-05-02</td>\n",
       "      <td>matt tuck</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Date        Name  Payment        Note\n",
       "0 2021-04-04    john doe  $100,50    Unhappy!\n",
       "1 2021-04-17    jane doe   $78.50  ?Satidfied\n",
       "2 2021-08-21  adam smith      $65     Neutral\n",
       "3 2021-05-02   matt tuck     $120    Unhappy-"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57482aab",
   "metadata": {},
   "source": [
    "## 数据类型的转换"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "de5aebb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    100.50\n",
       "1     78.50\n",
       "2        65\n",
       "3       120\n",
       "Name: Payment, dtype: object"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Payment'].str[0]\n",
    "df['Payment'].str[1:]\n",
    "df['Payment'].str[1:].str.replace(',','.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "010ee129",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T03:11:26.841299Z",
     "start_time": "2022-05-13T03:11:26.827821Z"
    }
   },
   "outputs": [],
   "source": [
    "# 支付Payment的数据类型是不能用于数值分析的。\n",
    "# 在将其转换为数字数据类型(即整数或浮点数)之前，我们需要删除美元符号并将第一行中的逗号替换为点。\n",
    "\n",
    "# 关于数值,建议统一为float数据格式,便于计算\n",
    "df['Payment'] = df['Payment'].str[1:].str.replace(',','.').astype('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "7a7e4261",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T03:11:33.736523Z",
     "start_time": "2022-05-13T03:11:33.708414Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-04-04</td>\n",
       "      <td>john doe</td>\n",
       "      <td>100.5</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-04-17</td>\n",
       "      <td>jane doe</td>\n",
       "      <td>78.5</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>adam smith</td>\n",
       "      <td>65.0</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-05-02</td>\n",
       "      <td>matt tuck</td>\n",
       "      <td>120.0</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Date        Name  Payment        Note\n",
       "0 2021-04-04    john doe    100.5    Unhappy!\n",
       "1 2021-04-17    jane doe     78.5  ?Satidfied\n",
       "2 2021-08-21  adam smith     65.0     Neutral\n",
       "3 2021-05-02   matt tuck    120.0    Unhappy-"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb34c877",
   "metadata": {},
   "source": [
    "##  删除冗余字符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "631efffd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T03:20:18.306002Z",
     "start_time": "2022-05-13T03:20:18.300787Z"
    }
   },
   "outputs": [],
   "source": [
    "# Note 列中的一些字符也需要删除。在处理大型数据集时，可能很难手动替换它们。\n",
    "# 我们可以做的是删除非字母数字字符（例如？、！、-、. 等）。在这种情况下也可以使用 replace 函数，因为它接受正则表达式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "f2a37fc2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T03:16:03.307592Z",
     "start_time": "2022-05-13T03:16:03.270108Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\10191977\\AppData\\Local\\Temp\\ipykernel_18416\\3524251737.py:2: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  df['Note'].str.replace('[^a-zA-Z]', ' ')  # ^a-zA-Z 表示非大小写字母\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0      Unhappy \n",
       "1     Satidfied\n",
       "2       Neutral\n",
       "3      Unhappy \n",
       "Name: Note, dtype: object"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 如果我们只想要字母字符：\n",
    "df['Note'].str.replace('[^a-zA-Z]', '')  # ^a-zA-Z 表示非大小写字母"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "3b15b9de",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T03:17:56.291522Z",
     "start_time": "2022-05-13T03:17:56.272486Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\10191977\\AppData\\Local\\Temp\\ipykernel_18416\\2278177357.py:2: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  df['Note'].str.replace('[^a-zA-Z0-9]', ' ')  # ^a-zA-Z 表示非大小写字母\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0      Unhappy \n",
       "1     Satidfied\n",
       "2       Neutral\n",
       "3      Unhappy \n",
       "Name: Note, dtype: object"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 如果我们想要字母和数字,我们需要在我们的正则表达式中添加数字：\n",
    "df['Note'].str.replace('[^a-zA-Z0-9]', ' ')  # ^a-zA-Z 表示非大小写字母"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "94ecd0d5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T03:19:40.120060Z",
     "start_time": "2022-05-13T03:19:40.097031Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\10191977\\AppData\\Local\\Temp\\ipykernel_18416\\3465157137.py:2: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  df['Note'] = df['Note'].str.replace('[^a-zA-Z]',' ').str.lower()\n"
     ]
    }
   ],
   "source": [
    "# 将结果统一为小写字母\n",
    "df['Note'] = df['Note'].str.replace('[^a-zA-Z]',' ').str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "d2f903d1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T03:19:45.827313Z",
     "start_time": "2022-05-13T03:19:45.787696Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-04-04</td>\n",
       "      <td>john doe</td>\n",
       "      <td>100.5</td>\n",
       "      <td>unhappy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-04-17</td>\n",
       "      <td>jane doe</td>\n",
       "      <td>78.5</td>\n",
       "      <td>satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>adam smith</td>\n",
       "      <td>65.0</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-05-02</td>\n",
       "      <td>matt tuck</td>\n",
       "      <td>120.0</td>\n",
       "      <td>unhappy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Date        Name  Payment        Note\n",
       "0 2021-04-04    john doe    100.5    unhappy \n",
       "1 2021-04-17    jane doe     78.5   satidfied\n",
       "2 2021-08-21  adam smith     65.0     neutral\n",
       "3 2021-05-02   matt tuck    120.0    unhappy "
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ffc58ec4",
   "metadata": {},
   "source": [
    "## 补充: 数值的替换(适用于空和非空)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db6d92ba",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T07:38:30.388530Z",
     "start_time": "2022-05-13T07:38:30.383770Z"
    }
   },
   "outputs": [],
   "source": [
    "# Pandas 中数据替换的方法，包含数值、文本、缺失值等替换，经常用于数据清洗整理，枚举转换，数据修正等情况。\n",
    "# Series 中的 replace() 和 DataFrame 中的 replace() 提供了一种高效而灵活的方法。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "d297bc49",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T07:38:39.508238Z",
     "start_time": "2022-05-13T07:38:39.462320Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed:0</th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "      <th>Remark</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>05/12/2021</td>\n",
       "      <td>James,Ben</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Neutral0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>2021-10-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$100</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed:0             Date        Name  Payment        Note  Remark\n",
       "0          0       04/04/2021    doe,john  $100,50    Unhappy!     NaN\n",
       "1          1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied     NaN\n",
       "2          2       2021-08-21  smith,Adam      $65     Neutral     NaN\n",
       "3          3       05/02/2021   Tuck,matt     $120    Unhappy-     NaN\n",
       "4          4       05/12/2021   James,Ben      NaN    Neutral0     NaN\n",
       "5          5       2021-10-11         NaN     $100     Neutral     NaN\n",
       "6          6              NaN         NaN      NaN         NaN     NaN"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'Unnamed:0': [0,1,2,3,4,5,6],\n",
    "                  'Date': ['04/04/2021','april 17th,2021','2021-08-21','05/02/2021','05/12/2021','2021-10-11',np.nan],\n",
    "                  'Name': ['doe,john','Doe,Jane','smith,Adam','Tuck,matt','James,Ben',np.nan,np.nan],\n",
    "                  'Payment': ['$100,50','$78.50','$65','$120',np.nan,'$100',np.nan],\n",
    "                  'Note': ['Unhappy!','?Satidfied','Neutral','Unhappy-','Neutral0','Neutral',np.nan],\n",
    "                  'Remark':[np.nan,np.nan,np.nan,np.nan,np.nan,np.nan,np.nan]},\n",
    "#                  index = [0,1,2,3,4,5]\n",
    "                 )\n",
    "df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "7c050452",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Name</th>\n",
       "      <th>Payment</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>04/04/2021</td>\n",
       "      <td>doe,john</td>\n",
       "      <td>$100,50</td>\n",
       "      <td>Unhappy!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>april 17th,2021</td>\n",
       "      <td>Doe,Jane</td>\n",
       "      <td>$78.50</td>\n",
       "      <td>?Satidfied</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-08-21</td>\n",
       "      <td>smith,Adam</td>\n",
       "      <td>$65</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>05/02/2021</td>\n",
       "      <td>Tuck,matt</td>\n",
       "      <td>$120</td>\n",
       "      <td>Unhappy-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05/12/2021</td>\n",
       "      <td>James,Ben</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Neutral0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2021-10-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$100</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Date        Name  Payment        Note\n",
       "0       04/04/2021    doe,john  $100,50    Unhappy!\n",
       "1  april 17th,2021    Doe,Jane   $78.50  ?Satidfied\n",
       "2       2021-08-21  smith,Adam      $65     Neutral\n",
       "3       05/02/2021   Tuck,matt     $120    Unhappy-\n",
       "4       05/12/2021   James,Ben      NaN    Neutral0\n",
       "5       2021-10-11         NaN     $100     Neutral"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop('Unnamed:0', axis=1, inplace=True)  \n",
    "df.dropna(axis=1,how='all', inplace=True)\n",
    "df.dropna(axis=0,how='all',inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ca21cf1",
   "metadata": {},
   "source": [
    "### 指定值替换"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "795c103a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T07:51:35.261571Z",
     "start_time": "2022-05-13T07:51:35.239983Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a  b\n",
       "0  0  0\n",
       "1  1  1\n",
       "2  2  2\n",
       "3  3  3\n",
       "4  4  4\n",
       "5  5  5"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'a':[0, 1, 2, 3, 4, 5],\n",
    "                  'b':[0, 1, 2, 3, 4, 5]})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "ed327d3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a  b\n",
       "0  0  0\n",
       "1  1  1\n",
       "2  2  2\n",
       "3  3  3\n",
       "4  4  4\n",
       "5  5  5"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将指定列的空值替换成指定值\n",
    "df.replace({'Payment': {np.nan: 100}}) # columns中,NAN替换为100\n",
    "df.replace({'columns': {10:100}}) # columns中,10替换为100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "d26c0ce3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T07:52:57.231406Z",
     "start_time": "2022-05-13T07:52:57.184869Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>400</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     a  b\n",
       "0  100  0\n",
       "1    1  1\n",
       "2    2  2\n",
       "3    3  3\n",
       "4  400  4\n",
       "5    5  5"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 以下是在 Series 中将 0 替换为 5：\n",
    "ser = pd.Series([0., 1., 2., 3., 4.])\n",
    "ser.replace(0, 5)\n",
    "\n",
    "# 也可以批量替换：\n",
    "ser.replace([0, 1, 2, 3, 4], [4, 3, 2, 1, 0])# 一一对应进行替换\n",
    "ser.replace({0: 10, 1: 100})# 用字典映射对应替换值\n",
    "\n",
    "df.replace({'a': 0, 'b': 5}, 100)# 将 a 列的 0 b 列中的 5 替换为 100\n",
    "df.replace({'a': {0: 100, 4: 400}})#  指定列里的替换规划"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0888610",
   "metadata": {},
   "source": [
    "### 向前向后填充"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "4a7eb5e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.0\n",
       "1    4.0\n",
       "2    4.0\n",
       "3    4.0\n",
       "4    4.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pad / ffill：向前填充\n",
    "# bfill / backfill：向后填充\n",
    "\n",
    "# 将 1，2，3 替换为它们前一个值\n",
    "ser.replace([1, 2, 3], method='pad') # ffill 是它同义词\n",
    "# 将 1，2，3 替换为它们后一个值\n",
    "ser.replace([1, 2, 3], method='bfill') # backfill 是它同义词"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cb207f5",
   "metadata": {},
   "source": [
    "### 正则表达式填充"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "497f3c37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a  b\n",
       "0  0  0\n",
       "1  1  1\n",
       "2  2  2\n",
       "3  3  3\n",
       "4  4  4\n",
       "5  5  5"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 如果遇到字符比较复杂的内容，就是使用正则（默认没有开启）进行匹配：\n",
    "\n",
    "df.replace(to_replace='bat', value='new') # 把 bat 替换为 new\n",
    "df.replace(to_replace=r'^ba.$', value='new', regex=True)# 利用正则将 ba 开头的替换为 new\n",
    "df.replace({'A': r'^ba.$'}, {'A': 'new'}, regex=True)# 如果多列规则不一的情况下可以按以下格式对应传入\n",
    "df.replace(regex=[r'^ba.$', 'foo'], value='new')# 多个规则替换为同一个值\n",
    "df.replace(regex={r'^ba.$': 'new', 'foo': 'xyz'})# 直接多个正则及对应的替换内容"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5966ded4",
   "metadata": {},
   "source": [
    "### 缺失值相关替换"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "6e16dc72",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-13T08:05:45.402853Z",
     "start_time": "2022-05-13T08:05:45.370087Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>b</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>.</td>\n",
       "      <td>d</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a  b    c\n",
       "0  0  a    a\n",
       "1  1  b    b\n",
       "2  2  .  NaN\n",
       "3  3  .    d"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = {'a': list(range(4)),\n",
    "     'b': list('ab..'),\n",
    "     'c': ['a', 'b', np.nan, 'd']\n",
    "    }\n",
    "# type(d)  # dirc\n",
    "df = pd.DataFrame(d)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "7e468d28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>d</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a   b    c\n",
       "0  0 NaN  NaN\n",
       "1  1 NaN  NaN\n",
       "2  2 NaN  NaN\n",
       "3  3 NaN    d"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.replace('.', np.nan)# 将.替换为 nan\n",
    "df.replace('.', np.nan)\n",
    "# 使用正则，将空格等替换为 nan\n",
    "df.replace(r'\\s*\\.\\s*', np.nan, regex=True)\n",
    "# 对应替换，a 换 b, 点换 nan\n",
    "df.replace(['a', '.'], ['b', np.nan])\n",
    "# 点换 dot, a 换 astuff(第一位+)\n",
    "df.replace([r'\\.', r'(a)'], ['dot', r'\\1stuff'], regex=True)\n",
    "# b 中的点要替换，替换为 b 替换规则为 nan，可以多列\n",
    "df.replace({'b': '.'}, {'b': np.nan})\n",
    "# 使用正则\n",
    "df.replace({'b': r'\\s*\\.\\s*'}, {'b': np.nan}, regex=True)\n",
    "# b列的 b 值换为空\n",
    "df.replace({'b': {'b': r''}}, regex=True)\n",
    "# b 列的点空格等换 nan\n",
    "df.replace(regex={'b': {r'\\s*\\.\\s*': np.nan}})\n",
    "# b列点等+ty\n",
    "df.replace({'b': r'\\s*(\\.)\\s*'},\n",
    "           {'b': r'\\1ty'},\n",
    "           regex=True)\n",
    "# 多个正则规则\n",
    "df.replace([r'\\s*\\.\\s*', r'a|b'], np.nan, regex=True)\n",
    "# 用参数名传参\n",
    "df.replace(regex=[r'\\s*\\.\\s*', r'a|b'], value=np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd69d785",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d102d8c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "llm",
   "language": "python",
   "name": "llm"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "284.75px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
